{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Tree.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/harvardnlp/pytorch-struct/blob/master/notebooks/BertDependencies.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dENII4iGN3S4",
        "colab_type": "code",
        "outputId": "e87f5c44-3da0-4df4-8f8b-f91c5adbb93e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 314
        }
      },
      "source": [
        "!pip install -qqq torchtext wandb\n",
        "!pip install -qqq pytorch-transformers\n",
        "!pip install -qqqU git+https://github.com/harvardnlp/pytorch-struct\n",
        "!git clone -q http://github.com/srush/temp"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 1.3MB 2.9MB/s \n",
            "\u001b[K     |████████████████████████████████| 102kB 25.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 235kB 44.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 92kB 25.6MB/s \n",
            "\u001b[K     |████████████████████████████████| 460kB 43.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 92kB 26.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 256kB 45.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 71kB 25.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 184kB 41.2MB/s \n",
            "\u001b[?25h  Building wheel for gql (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for shortuuid (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for watchdog (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "  Building wheel for torch-struct (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "fatal: destination path 'temp' already exists and is not an empty directory.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EghAJ7aJDl4V",
        "colab_type": "code",
        "outputId": "970f7f28-6f4a-4c80-f4c8-3b348bdbbfd7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "import torchtext\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch_struct import DependencyCRF\n",
        "import torch_struct.data\n",
        "import torchtext.data as data\n",
        "from pytorch_transformers import AdamW, WarmupLinearSchedule\n",
        "from pytorch_transformers import *\n",
        "\n",
        "config = {\"bert\": \"bert-base-cased\", \"H\" : 768, \"dropout\": 0.2\n",
        "         }\n",
        "\n",
        "# Comment or add your wandb\n",
        "import wandb\n",
        "wandb.init(project=\"pytorch-struct-tagging\", config=config)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://app.wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "wandb: ERROR Not authenticated.  Copy a key from https://app.wandb.ai/authorize\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "API Key: ··········\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "wandb: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "            Notebook configured with <a href=\"https://wandb.com\" target=\"_blank\">W&B</a>. You can <a href=\"https://app.wandb.ai/srush/pytorch-struct-tagging/runs/j1dqjny2\" target=\"_blank\">open</a> the run page, or call <code>%%wandb</code>\n",
              "            in a cell containing your training loop to display live results.  Learn more in our <a href=\"https://docs.wandb.com/docs/integrations/jupyter.html\" target=\"_blank\">docs</a>.\n",
              "        "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "W&B Run: https://app.wandb.ai/srush/pytorch-struct-tagging/runs/j1dqjny2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l63Zy6ap95Ug",
        "colab_type": "text"
      },
      "source": [
        "Parse the conll dependency data."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vQoaK1lv-CXc",
        "colab_type": "text"
      },
      "source": [
        "TorchText batching setup."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YsKQXHflOmDe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0d184e8e-a2a4-4c5a-830d-3d0bc96e3963"
      },
      "source": [
        "model_class, tokenizer_class, pretrained_weights = BertModel, BertTokenizer, config['bert']\n",
        "tokenizer = tokenizer_class.from_pretrained(pretrained_weights)\n",
        "def batch_num(nums):\n",
        "    lengths = torch.tensor([len(n) for n in nums]).long()\n",
        "    n = lengths.max()\n",
        "    out = torch.zeros(len(nums), n).long()\n",
        "    for b, n in enumerate(nums):\n",
        "        out[b, :len(n)] = torch.tensor(n)\n",
        "    return out, lengths\n",
        "HEAD = data.RawField(preprocessing= lambda x: [int(i) for i in x],\n",
        "                     postprocessing=batch_num)\n",
        "WORD = torch_struct.data.SubTokenizedField(tokenizer)\n",
        "HEAD.is_target = True\n",
        "train = torch_struct.data.ConllXDataset(\"temp/wsj.train.conllx\", (('word', WORD), ('head', HEAD)),\n",
        "                     filter_pred=lambda x: 5 < len(x.word[0]) < 40)\n",
        "train_iter = torch_struct.data.TokenBucket(train, 750)\n",
        "val = torch_struct.data.ConllXDataset(\"temp/wsj.dev.conllx\", (('word', WORD), ('head', HEAD)),\n",
        "                     filter_pred=lambda x: 5 < len(x.word[0]) < 40)\n",
        "val_iter = torchtext.data.BucketIterator(val, \n",
        "    batch_size=20,\n",
        "    device=\"cuda:0\")"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 213450/213450 [00:00<00:00, 385689.56B/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z-40o2UA-Ha7",
        "colab_type": "text"
      },
      "source": [
        "Make a Bert model to compute the potentials"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QUKVqGUsSmYY",
        "colab_type": "code",
        "outputId": "6317e23a-5ac7-45f2-9d01-2249d9675b2c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 361
        }
      },
      "source": [
        "H = config[\"H\"]\n",
        "class Model(nn.Module):\n",
        "    def __init__(self, hidden):\n",
        "        super().__init__()\n",
        "        self.base_model = model_class.from_pretrained(pretrained_weights)\n",
        "        self.linear = nn.Linear(H, H)\n",
        "        self.bilinear = nn.Linear(H, H)\n",
        "        self.root = nn.Parameter(torch.rand(H))\n",
        "        self.dropout = nn.Dropout(config[\"dropout\"])\n",
        "        \n",
        "    def forward(self, words, mapper):\n",
        "        out = self.dropout(self.base_model(words)[0])\n",
        "        out = torch.einsum(\"bca,bch->bah\", mapper.float().cuda(), out)\n",
        "        final2 = torch.einsum(\"bnh,hg->bng\", out, self.linear.weight)\n",
        "        final = torch.einsum(\"bnh,hg,bmg->bnm\", out, self.bilinear.weight, final2)\n",
        "        root_score = torch.einsum(\"bnh,h->bn\", out, self.root)\n",
        "        final = final[:, 1:-1, 1:-1]\n",
        "        N = final.shape[1]\n",
        "        final[:, torch.arange(N), torch.arange(N)] += root_score[:, 1:-1]\n",
        "        return final\n",
        "\n",
        "model = Model(H)\n",
        "wandb.watch(model)\n",
        "model.cuda()\n",
        "None"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-18-df3bd0213a76>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mH\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0mwandb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/wandb/__init__.py\u001b[0m in \u001b[0;36mwatch\u001b[0;34m(models, criterion, log, log_freq)\u001b[0m\n\u001b[1;32m    121\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mwatch_called\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m         raise ValueError(\n\u001b[0;32m--> 123\u001b[0;31m             \u001b[0;34m\"You can only call `wandb.watch` once per process. If you want to watch multiple models, pass them in as a tuple.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    124\u001b[0m         )\n\u001b[1;32m    125\u001b[0m     \u001b[0mwatch_called\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: You can only call `wandb.watch` once per process. If you want to watch multiple models, pass them in as a tuple."
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "grxGRu2b-LdT",
        "colab_type": "text"
      },
      "source": [
        "Generic training loop. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HENkCYOKSwHr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def validate():\n",
        "    incorrect_edges = 0\n",
        "    total_edges = 0\n",
        "    model.eval()\n",
        "    for i, ex in enumerate(val_iter):\n",
        "        words, mapper, _ = ex.word\n",
        "        label, lengths = ex.head\n",
        "        batch, _ = label.shape\n",
        "\n",
        "        final = model(words.cuda(), mapper)\n",
        "        for b in range(batch):\n",
        "            final[b, lengths[b]-1:, :] = 0\n",
        "            final[b, :, lengths[b]-1:] = 0\n",
        "        dist = DependencyCRF(final, lengths=lengths)\n",
        "        argmax = dist.argmax\n",
        "        gold = dist.struct.to_parts(label, lengths=lengths).type_as(argmax)\n",
        "        incorrect_edges += (out[:, :].cpu() - gold[:, :].cpu()).abs().sum() / 2.0\n",
        "        total_edges += gold.sum()\n",
        "\n",
        "    print(total_edges, incorrect_edges)   \n",
        "    model.train()\n",
        "\n",
        "def train(train_iter, val_iter, model):\n",
        "    opt = AdamW(model.parameters(), lr=1e-4, eps=1e-8)\n",
        "    scheduler = WarmupLinearSchedule(opt, warmup_steps=20, t_total=2500)\n",
        "    model.train()\n",
        "    losses = []\n",
        "    for i, ex in enumerate(train_iter):\n",
        "        opt.zero_grad()\n",
        "        words, mapper, _ = ex.word\n",
        "        label, lengths = ex.head\n",
        "        batch, _ = label.shape\n",
        "        \n",
        "        # Model\n",
        "        final = model(words.cuda(), mapper)\n",
        "        for b in range(batch):\n",
        "            final[b, lengths[b]-1:, :] = 0\n",
        "            final[b, :, lengths[b]-1:] = 0\n",
        "        \n",
        "        if not lengths.max() <= final.shape[1] + 1:\n",
        "            print(\"fail\")\n",
        "            continue\n",
        "        dist = DependencyCRF(final, lengths=lengths)\n",
        "\n",
        "        labels = dist.struct.to_parts(label, lengths=lengths).type_as(final)\n",
        "        log_prob = dist.log_prob(labels)\n",
        "\n",
        "        loss = log_prob.sum()\n",
        "        (-loss).backward()\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        opt.step()\n",
        "        scheduler.step()\n",
        "        losses.append(loss.detach())\n",
        "        if i % 50 == 1:            \n",
        "            print(-torch.tensor(losses).mean(), words.shape)\n",
        "            losses = []\n",
        "        if i % 600 == 500:\n",
        "            validate(val_iter)        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D-HW3z1VT2MG",
        "colab_type": "code",
        "outputId": "0d7b9ef1-8c82-4c8b-c25b-4737237e02ce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 335
        }
      },
      "source": [
        "train(train_iter, val_iter, model)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-5056f3e28c15>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-16-f12d56147d6f>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(train_iter, val_iter, model)\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlog_prob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m         \u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip_grad_norm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    116\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m         \"\"\"\n\u001b[0;32m--> 118\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     91\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     92\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [24, 29, 29]], which is output 0 of SliceBackward, is at version 49; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True)."
          ]
        }
      ]
    }
  ]
}